package spider.web;

import org.apache.commons.httpclient.DefaultHttpMethodRetryHandler;
import org.apache.commons.httpclient.HttpClient;
import org.apache.commons.httpclient.HttpException;
import org.apache.commons.httpclient.HttpStatus;
import org.apache.commons.httpclient.methods.GetMethod;
import org.apache.commons.httpclient.params.HttpMethodParams;

import java.io.DataOutputStream;
import java.io.File;
import java.io.FileOutputStream;
import java.io.IOException;
import java.util.regex.Matcher;
import java.util.regex.Pattern;

/**
 * @author: zhangzeli
 * @date 19:20 2018/5/4
 * <P></P>
 */
public class DownLoadFile {

    //根据url和网页类型生成需要保存的网页文件名,去除url中的非文件名字符串
    public String getFileNameByUrl(String url,String contentType){
        //移除http
        url = url.substring(7);
        //text/html 类型
        if(contentType.indexOf("html")!=-1){
            url = url.replaceAll("[\\?/:*|]","_")+".html";
            return url;
        }else{
            return url.replaceAll("[\\?/:*|<>\"]","_")+"."+contentType.substring(contentType.lastIndexOf("/")+1);
        }
    }

    /**
     * 保存网页字节数组到本地文件
     * @param data
     * @param filePath
     */
    public void saveToLoacl(byte[] data,String filePath){
        try {
            DataOutputStream out =new DataOutputStream(new FileOutputStream(new File(filePath)));
            for(int i=0;i<data.length;i++){
                out.write(data[i]);
            }
            out.flush();
            out.close();
        }catch (IOException E){
            E.printStackTrace();
        }
    }

    //下载UTL指向的网页
    public String downloadFile(String url){
        String filePath ="";
        //1,生成HttpClient对象并设置参数
        HttpClient httpClient = new HttpClient();
        //设置http连接超时5s
        httpClient.getHttpConnectionManager().getParams().setConnectionTimeout(5000);
        //2,生成GetMethod 对象并设置参数
        GetMethod getMethod = new GetMethod(url);
        //设置get请求超时5s
        getMethod.getParams().setParameter(HttpMethodParams.SO_TIMEOUT,5000);
        //设置请求重试处理
        getMethod.getParams().setParameter(HttpMethodParams.RETRY_HANDLER,new DefaultHttpMethodRetryHandler());
        //3,执行http get
        try {
            int statusCode = httpClient.executeMethod(getMethod);
            if(statusCode!= HttpStatus.SC_OK){
                System.out.println("method failed:"+getMethod.getStatusLine());
                filePath ="";
            }
            //处理相应内容
            byte[] responseBody =getMethod.getResponseBody();//读取字节数组
            //根据网页url生成保存时的文件名
            filePath="E:\\ideawork\\maven\\spiderhtml\\"+getFileNameByUrl(url,getMethod.getResponseHeader("Content-Type").getValue());
            saveToLoacl(responseBody,filePath);
        }catch (HttpException e){
            System.out.println("Please check your provided http address!");
            e.printStackTrace();
        }catch (IOException e){
            e.printStackTrace();
        }finally {
            getMethod.releaseConnection();
        }
        return filePath;
    }
}
